In [22]:
%matplotlib nbagg

In [23]:
from iuvs import io

In [2]:
df = pd.read_hdf('/home/klay6683/to_keep/l1a_dark_scan.h5', 'df')
# df = pd.read_hdf('/Users/klay6683/data/iuvs/l1a_dark_scan.h5', 'df')

using previously defined clean up function


In [3]:
from iuvs import meta


/usr/local/python3/miniconda/lib/python3.4/site-packages/IPython/kernel/__init__.py:13: ShimWarning: The `IPython.kernel` package has been deprecated. You should import from ipykernel or jupyter_client instead.
  "You should import from ipykernel or jupyter_client instead.", ShimWarning)

In [4]:
df = meta.clean_up_dark_scan(df)

creating little checking function:


In [5]:
def check_values(col):
    return df[col.upper()].value_counts(dropna=False)

filtering

Filtering out cruise and transition and integrations other than 14.4, 1.4, 4.2


In [6]:
df = df[df.COLLECTION_ID!='cruise']
df = df[df.COLLECTION_ID!='transition']
df = df[df.INT_TIME.isin([14400, 4200, 1400])]

Pick time resolution for resampling


In [7]:
timeres = '5s'

HK reading


In [8]:
hk = pd.read_hdf('/home/klay6683/to_keep/HK_DB.h5')

In [10]:
hk2015 = hk['2015']

In [12]:
hk2015 = hk2015.filter(regex='DET_TEMP_C')

In [13]:
hk2015.info()


<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 597185 entries, 2015-01-01 00:00:21.165966331 to 2015-08-11 19:30:57.949607491
Data columns (total 2 columns):
FUV_DET_TEMP_C    597185 non-null float64
MUV_DET_TEMP_C    597185 non-null float64
dtypes: float64(2)
memory usage: 13.7 MB

In [21]:
hk2015.iloc[:10].resample(timeres)


Out[21]:
FUV_DET_TEMP_C MUV_DET_TEMP_C
2015-01-01 00:00:20 -18.005027 -19.256848
2015-01-01 00:00:25 NaN NaN
2015-01-01 00:00:30 NaN NaN
2015-01-01 00:00:35 NaN NaN
2015-01-01 00:00:40 NaN NaN
2015-01-01 00:00:45 NaN NaN
2015-01-01 00:00:50 -18.019747 -19.264394
2015-01-01 00:00:55 NaN NaN
2015-01-01 00:01:00 NaN NaN
2015-01-01 00:01:05 NaN NaN
2015-01-01 00:01:10 NaN NaN
2015-01-01 00:01:15 NaN NaN
2015-01-01 00:01:20 -18.041835 -19.279491
2015-01-01 00:01:25 NaN NaN
2015-01-01 00:01:30 NaN NaN
2015-01-01 00:01:35 NaN NaN
2015-01-01 00:01:40 NaN NaN
2015-01-01 00:01:45 NaN NaN
2015-01-01 00:01:50 -18.056565 -19.294592
2015-01-01 00:01:55 NaN NaN
2015-01-01 00:02:00 NaN NaN
2015-01-01 00:02:05 NaN NaN
2015-01-01 00:02:10 NaN NaN
2015-01-01 00:02:15 NaN NaN
2015-01-01 00:02:20 -18.071300 -19.324808
2015-01-01 00:02:25 NaN NaN
2015-01-01 00:02:30 NaN NaN
2015-01-01 00:02:35 NaN NaN
2015-01-01 00:02:40 NaN NaN
2015-01-01 00:02:45 NaN NaN
2015-01-01 00:02:50 -18.086039 -19.332365
2015-01-01 00:02:55 NaN NaN
2015-01-01 00:03:00 NaN NaN
2015-01-01 00:03:05 NaN NaN
2015-01-01 00:03:10 NaN NaN
2015-01-01 00:03:15 NaN NaN
2015-01-01 00:03:20 -18.100783 -19.347483
2015-01-01 00:03:25 NaN NaN
2015-01-01 00:03:30 NaN NaN
2015-01-01 00:03:35 NaN NaN
2015-01-01 00:03:40 NaN NaN
2015-01-01 00:03:45 NaN NaN
2015-01-01 00:03:50 -18.115531 -19.370167
2015-01-01 00:03:55 NaN NaN
2015-01-01 00:04:00 NaN NaN
2015-01-01 00:04:05 NaN NaN
2015-01-01 00:04:10 NaN NaN
2015-01-01 00:04:15 NaN NaN
2015-01-01 00:04:20 -18.130283 -19.385296
2015-01-01 00:04:25 NaN NaN
2015-01-01 00:04:30 NaN NaN
2015-01-01 00:04:35 NaN NaN
2015-01-01 00:04:40 NaN NaN
2015-01-01 00:04:45 NaN NaN
2015-01-01 00:04:50 -18.145040 -19.400430

In [9]:
hktemps = hk['2015'].filter(regex='DET_TEMP_C').resample(timeres, how='ffill').dropna(how='all')


---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-9-89719c6a4669> in <module>()
----> 1 hktemps = hk['2015'].filter(regex='DET_TEMP_C').resample(timeres, how='ffill').dropna(how='all')

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/generic.py in resample(self, rule, how, axis, fill_method, closed, label, convention, kind, loffset, limit, base)
   3484                               fill_method=fill_method, convention=convention,
   3485                               limit=limit, base=base)
-> 3486         return sampler.resample(self).__finalize__(self)
   3487 
   3488     def first(self, offset):

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/tseries/resample.py in resample(self, obj)
     80 
     81         if isinstance(ax, DatetimeIndex):
---> 82             rs = self._resample_timestamps()
     83         elif isinstance(ax, PeriodIndex):
     84             offset = to_offset(self.freq)

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/tseries/resample.py in _resample_timestamps(self, kind)
    313             # Irregular data, have to use groupby
    314             grouped = obj.groupby(grouper, axis=self.axis)
--> 315             result = grouped.aggregate(self._agg_method)
    316 
    317             if self.fill_method is not None:

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in aggregate(self, arg, *args, **kwargs)
   2784     def aggregate(self, arg, *args, **kwargs):
   2785         if isinstance(arg, compat.string_types):
-> 2786             return getattr(self, arg)(*args, **kwargs)
   2787 
   2788         result = OrderedDict()

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in wrapper(*args, **kwargs)
    584 
    585             try:
--> 586                 return self.apply(curried_with_axis)
    587             except Exception:
    588                 try:

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in apply(self, func, *args, **kwargs)
    691         # ignore SettingWithCopy here in case the user mutates
    692         with option_context('mode.chained_assignment',None):
--> 693             return self._python_apply_general(f)
    694 
    695     def _python_apply_general(self, f):

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in _python_apply_general(self, f)
    695     def _python_apply_general(self, f):
    696         keys, values, mutated = self.grouper.apply(f, self._selected_obj,
--> 697                                                    self.axis)
    698 
    699         return self._wrap_applied_output(keys, values,

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in apply(self, f, data, axis)
   1764         result_values = []
   1765         mutated = False
-> 1766         for key, group in self.get_iterator(data, axis=axis):
   1767             object.__setattr__(group, 'name', key)
   1768 

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in get_iterator(self, data, axis)
   1754         for edge, label in zip(self.bins, self.binlabels):
   1755             if label is not tslib.NaT:
-> 1756                 yield label, slicer(start,edge)
   1757             start = edge
   1758 

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/groupby.py in <lambda>(start, edge)
   1745         """
   1746         if isinstance(data, NDFrame):
-> 1747             slicer = lambda start,edge: data._slice(slice(start,edge),axis=axis)
   1748             length = len(data.axes[axis])
   1749         else:

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/generic.py in _slice(self, slobj, axis, kind)
   1193         """
   1194         axis = self._get_block_manager_axis(axis)
-> 1195         result = self._constructor(self._data.get_slice(slobj, axis=axis))
   1196         result = result.__finalize__(self)
   1197 

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/internals.py in get_slice(self, slobj, axis)
   2696 
   2697         bm = self.__class__(new_blocks, new_axes, do_integrity_check=False,
-> 2698                             fastpath=True)
   2699         bm._consolidate_inplace()
   2700         return bm

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/internals.py in __init__(self, blocks, axes, do_integrity_check, fastpath)
   2205         self._consolidate_check()
   2206 
-> 2207         self._rebuild_blknos_and_blklocs()
   2208 
   2209     def make_empty(self, axes=None):

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/internals.py in _rebuild_blknos_and_blklocs(self)
   2295             new_blklocs[rl.indexer] = np.arange(len(rl))
   2296 
-> 2297         if (new_blknos == -1).any():
   2298             raise AssertionError("Gaps in blk ref_locs")
   2299 

KeyboardInterrupt: 

Grouping by bin set


In [ ]:
g = df.groupby(['CHANNEL', 'INT_TIME', 'activity', 'BINNING_SET'])

In [ ]:
list_g = list(g)

In [ ]:
list_g[0][1]

In [ ]:
testdf = list_g[0][1].resample(timeres)

In [ ]:
hktemps['2015-01-04 14:35']

In [55]:
for name, group in g:
    print(name, "Size:", len(group))


('FUV', 1400, 'outbound', (0, 3, 880, 3)) Size: 2
('FUV', 1400, 'outbound', (52, 4, 89, 115)) Size: 233
('FUV', 1400, 'outbound', (52, 8, 89, 115)) Size: 490
('FUV', 1400, 'outlimb', (0, 4, 89, 115)) Size: 4
('FUV', 1400, 'outlimb', (52, 4, 89, 115)) Size: 60
('FUV', 1400, 'outlimb', (52, 8, 89, 115)) Size: 360
('FUV', 4200, 'outboundhifi', (0, 1, 319, 11)) Size: 98
('FUV', 4200, 'outdiskhifi', (0, 1, 319, 11)) Size: 18
('FUV', 4200, 'periapse', (0, 1, 0, 1)) Size: 16
('FUV', 4200, 'periapse', (0, 4, 89, 115)) Size: 8484
('FUV', 4200, 'periapse', (1, 4, 89, 115)) Size: 355
('FUV', 4200, 'periapsehifi', (0, 1, 319, 11)) Size: 10
('FUV', 14400, 'apoapse', (0, 1, 0, 1)) Size: 14
('FUV', 14400, 'apoapse', (0, 4, 89, 80)) Size: 2372
('FUV', 14400, 'apoapse', (1, 4, 89, 80)) Size: 514
('FUV', 14400, 'apoapse', (52, 8, 89, 80)) Size: 5468
('FUV', 14400, 'apoapse', (65, 4, 89, 80)) Size: 100
('FUV', 14400, 'inbound', (0, 1, 0, 1)) Size: 4
('FUV', 14400, 'inbound', (0, 4, 89, 115)) Size: 6
('FUV', 14400, 'inbound', (52, 4, 89, 115)) Size: 701
('FUV', 14400, 'inbound', (52, 8, 89, 115)) Size: 1090
('FUV', 14400, 'outbound', (0, 1, 0, 1)) Size: 4
('FUV', 14400, 'outbound', (52, 4, 89, 115)) Size: 354
('FUV', 14400, 'outbound', (52, 8, 89, 115)) Size: 74
('FUV', 14400, 'outboundhifi', (0, 1, 319, 11)) Size: 89
('FUV', 14400, 'outdisk', (0, 1, 0, 1)) Size: 8
('FUV', 14400, 'outdisk', (0, 4, 89, 115)) Size: 4
('FUV', 14400, 'outdisk', (52, 4, 89, 115)) Size: 88
('FUV', 14400, 'outdisk', (52, 8, 89, 115)) Size: 472
('MUV', 1400, 'apoapse', (0, 16, 0, 16)) Size: 1
('MUV', 1400, 'inbound', (0, 16, 0, 16)) Size: 1
('MUV', 1400, 'outbound', (0, 3, 880, 3)) Size: 2
('MUV', 1400, 'outbound', (0, 4, 96, 115)) Size: 100
('MUV', 1400, 'outbound', (2, 34, 96, 115)) Size: 133
('MUV', 1400, 'outbound', (140, 8, 96, 115)) Size: 490
('MUV', 1400, 'outlimb', (0, 4, 96, 115)) Size: 64
('MUV', 1400, 'outlimb', (140, 8, 96, 115)) Size: 360
('MUV', 4200, 'inbound', (0, 16, 0, 16)) Size: 2
('MUV', 4200, 'outboundhifi', (0, 1, 441, 11)) Size: 97
('MUV', 4200, 'outdiskhifi', (0, 1, 441, 11)) Size: 18
('MUV', 4200, 'periapse', (0, 1, 0, 1)) Size: 16
('MUV', 4200, 'periapse', (0, 4, 96, 115)) Size: 4030
('MUV', 4200, 'periapse', (140, 4, 96, 115)) Size: 4830
('MUV', 4200, 'periapsehifi', (0, 1, 441, 11)) Size: 10
('MUV', 14400, 'apoapse', (0, 1, 0, 1)) Size: 14
('MUV', 14400, 'apoapse', (0, 4, 101, 80)) Size: 380
('MUV', 14400, 'apoapse', (0, 16, 0, 16)) Size: 1
('MUV', 14400, 'apoapse', (2, 34, 101, 80)) Size: 2740
('MUV', 14400, 'apoapse', (140, 8, 101, 80)) Size: 5334
('MUV', 14400, 'inbound', (0, 1, 0, 1)) Size: 4
('MUV', 14400, 'inbound', (0, 16, 0, 16)) Size: 103
('MUV', 14400, 'inbound', (2, 34, 101, 80)) Size: 678
('MUV', 14400, 'outbound', (0, 1, 0, 1)) Size: 4
('MUV', 14400, 'outbound', (0, 4, 96, 115)) Size: 204
('MUV', 14400, 'outbound', (0, 16, 0, 16)) Size: 2
('MUV', 14400, 'outbound', (2, 34, 96, 115)) Size: 150
('MUV', 14400, 'outbound', (140, 8, 96, 115)) Size: 74
('MUV', 14400, 'outboundhifi', (0, 1, 441, 11)) Size: 88
('MUV', 14400, 'outdisk', (0, 1, 0, 1)) Size: 8
('MUV', 14400, 'outdisk', (0, 4, 96, 115)) Size: 92
('MUV', 14400, 'outdisk', (140, 8, 96, 115)) Size: 472

In [ ]:


In [44]:
muv1min = df[df.CHANNEL=='MUV'].resample('5s').dropna(how='all')
fuv1min = df[df.CHANNEL=='FUV'].resample('5s').dropna(how='all')

In [45]:
muv1min['HK_MUV_TEMP'] = hktemps.MUV_DET_TEMP_C
fuv1min['HK_FUV_TEMP'] = hktemps.FUV_DET_TEMP_C

In [57]:
for item in [muv1min, fuv1min]:
    item.loc[item['INT_TIME']<1.4, 'INT_TIME'] = 1.4

In [21]:
import seaborn as sns


/usr/local/python3/miniconda/lib/python3.4/site-packages/matplotlib-1.5.dev1-py3.4-linux-x86_64.egg/matplotlib/__init__.py:866: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))

In [22]:
sns.set_style('whitegrid')
sns.set_context('talk')

In [54]:
df.INT_TIME.value_counts()


Out[54]:
14.4    21710
4.2     17984
1.4      2300
Name: INT_TIME, dtype: int64

In [87]:
g = sns.FacetGrid(muv1min, col='INT_TIME', size=4)
g.map(plt.scatter, 'DET_TEMP', 'mean')


/usr/local/python3/miniconda/lib/python3.4/site-packages/matplotlib-1.5.dev1-py3.4-linux-x86_64.egg/matplotlib/__init__.py:886: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))
Out[87]:
<seaborn.axisgrid.FacetGrid at 0x7f85d2724e48>

In [88]:
g = sns.FacetGrid(muv1min, col='INT_TIME', size=4, ylim=(0,1))
g.map(plt.scatter, 'DET_TEMP', 'mean')


/usr/local/python3/miniconda/lib/python3.4/site-packages/matplotlib-1.5.dev1-py3.4-linux-x86_64.egg/matplotlib/__init__.py:886: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))
Out[88]:
<seaborn.axisgrid.FacetGrid at 0x7f85d2865f60>

In [91]:
g = sns.FacetGrid(muv1min, col='INT_TIME', size=4, ylim=(0,1))
g.map(plt.scatter, 'HK_MUV_TEMP', 'mean')


/usr/local/python3/miniconda/lib/python3.4/site-packages/matplotlib-1.5.dev1-py3.4-linux-x86_64.egg/matplotlib/__init__.py:886: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))
Out[91]:
<seaborn.axisgrid.FacetGrid at 0x7f85e808b6a0>

In [94]:
g = sns.FacetGrid(muv1min, col='BINNING_SET', col_wrap=3 )
g.map(plt.scatter, 'HK_MUV_TEMP', 'mean')


/usr/local/python3/miniconda/lib/python3.4/site-packages/matplotlib-1.5.dev1-py3.4-linux-x86_64.egg/matplotlib/__init__.py:886: UserWarning: axes.color_cycle is deprecated and replaced with axes.prop_cycle; please use the latter.
  warnings.warn(self.msg_depr % (key, alt_key))
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
<ipython-input-94-f152be247990> in <module>()
----> 1 g = sns.FacetGrid(muv1min, col='BINNING_SET', col_wrap=3 )
      2 g.map(plt.scatter, 'HK_MUV_TEMP', 'mean')

/usr/local/python3/miniconda/lib/python3.4/site-packages/seaborn/axisgrid.py in __init__(self, data, row, col, hue, col_wrap, sharex, sharey, size, aspect, palette, row_order, col_order, hue_order, hue_kws, dropna, legend_out, despine, margin_titles, xlim, ylim, subplot_kws, gridspec_kws)
    244             col_names = []
    245         else:
--> 246             col_names = utils.categorical_order(data[col], col_order)
    247 
    248         # Additional dict of kwarg -> list of values for mapping the hue var

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/frame.py in __getitem__(self, key)
   1897             return self._getitem_multilevel(key)
   1898         else:
-> 1899             return self._getitem_column(key)
   1900 
   1901     def _getitem_column(self, key):

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/frame.py in _getitem_column(self, key)
   1904         # get column
   1905         if self.columns.is_unique:
-> 1906             return self._get_item_cache(key)
   1907 
   1908         # duplicate columns & possible reduce dimensionaility

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/generic.py in _get_item_cache(self, item)
   1086         res = cache.get(item)
   1087         if res is None:
-> 1088             values = self._data.get(item)
   1089             res = self._box_item_values(item, values)
   1090             cache[item] = res

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/internals.py in get(self, item, fastpath)
   2884 
   2885             if not isnull(item):
-> 2886                 loc = self.items.get_loc(item)
   2887             else:
   2888                 indexer = np.arange(len(self.items))[isnull(self.items)]

/usr/local/python3/miniconda/lib/python3.4/site-packages/pandas-0.16.2+535.gf82e177-py3.4-linux-x86_64.egg/pandas/core/index.py in get_loc(self, key, method, tolerance)
   1668                 raise ValueError('tolerance argument only valid if using pad, '
   1669                                  'backfill or nearest lookups')
-> 1670             return self._engine.get_loc(_values_from_object(key))
   1671 
   1672         indexer = self.get_indexer([key], method=method,

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:3979)()

pandas/index.pyx in pandas.index.IndexEngine.get_loc (pandas/index.c:3843)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12265)()

pandas/hashtable.pyx in pandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:12216)()

KeyError: 'BINNING_SET'

In [ ]:
g = sns.FacetGrid(df, col='CHANNEL')
g.map(plt.scatter, )

In [93]:
g?

In [ ]:


In [ ]:


In [ ]:
plt.figure()
sns.boxplot(x='COLLECTION_ID', y='mean', hue='CHANNEL', data=df)

In [ ]:
plt.figure()
sns.violinplot(x='COLLECTION_ID', y='median', hue='CHANNEL', data=df,
               split=True, cut=0)

In [ ]:
plt.figure()
sns.violinplot(x='COLLECTION_ID', y='median', data=df, hue='CHANNEL', inner=None,
               bw=.1, scale='count', scale_hue=False, cut=0)
sns.stripplot(x='COLLECTION_ID', y='median', data=df, jitter=True, size=3)

In [ ]:
plt.figure()
sns.barplot(x='COLLECTION_ID', y='mean', hue='CHANNEL', data=df)

In [ ]:
df = df[df.INT_TIME.isin([1.4, 4.2, 14.4])]

In [ ]:
df = df.sort_values(by='INT_TIME')

In [ ]:
plt.figure()
sns.pointplot(hue='COLLECTION_ID', y='mean', x='INT_TIME', data=df)

In [ ]:
%matplotlib inline

In [ ]:
df.columns

In [ ]:
to_drop = []
for col in df:
    n_unique = len(df[col].unique())
    if n_unique == 1:
        print(col)
        print(df[col].unique())
        to_drop.append(col)
df = df.drop(to_drop, axis=1)

In [ ]:
df.INT_TIME.value_counts()/df.INT_TIME.size

In [ ]:
sns.lmplot(x='DET_TEMP', y='median', hue='CHANNEL',
           col='INT_TIME', row='COLLECTION_ID', data=df, size=4)

In [ ]:
g = sns.FacetGrid(df, row='INT_TIME', hue='CHANNEL', size=1.7, aspect=4,)
g.map(sns.distplot, 'mean')

In [ ]:
df.BINNING_SET = df.BINNING_SET.astype('category')

In [ ]:
sns.jointplot(x='DET_TEMP',y='mean', data=df[(df.CHANNEL=='MUV') & (df.INT_TIME==14.4)])

In [ ]:
s = df.PRODUCT_ID.str.split('_')
s = s.map(lambda x: x[3]).str.split('-')
df['activity'] = s.map(lambda x: x[0])

In [ ]:
sns.lmplot(x='DET_TEMP', y='mean',hue='CHANNEL', data=df, aspect=1.33, size=8)

In [ ]:
ax = sns.lmplot(x='DET_TEMP', y='mean',hue='CHANNEL', data=df[df.INT_TIME==14.4],
           aspect=1.33, size=6)
plt.gcf().tight_layout()

In [ ]:
df.groupby(['INT_TIME','BINNING_SET']).size()

In [ ]:
df.columns

In [ ]:
sns.set()
sns.pairplot(df['channel case_temp det_temp int_time solar_longitude'.upper().split() + ['mean']].dropna(),
             hue='INT_TIME')

In [ ]:


In [ ]:


In [ ]:


In [ ]:
df[df['mean']>1.0].COLLECTION_ID.value_counts()

In [ ]:
df.COLLECTION_ID.value_counts()

In [ ]:
sns.set_context('talk')

In [ ]:
plt.figure()
ax = sns.stripplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
              jitter=True)
ax.set_title("Dark mean vs INT_TIME")

In [ ]:
plt.figure()
ax = sns.boxplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
                )
ax.set_title("Dark mean vs INT_TIME")

In [ ]:
plt.figure()
ax = sns.boxplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
                )
ax.set_title("Dark mean vs INT_TIME")

In [ ]:
plt.figure()
ax = sns.violinplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
                   split=True, cut=0, scale='area')
ax.set_title("Dark mean vs INT_TIME")

In [ ]:
sns.violinplot?

In [ ]:
plt.figure()
ax = sns.violinplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
                   split=True, cut=0, scale='count')
sns.stripplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
              jitter=True, size=4)
ax.set_title("Dark mean vs INT_TIME")

In [ ]:
sns.violinplot?

In [ ]:
plt.figure()
sns.distplot(df.loc[(df.INT_TIME==4.2) & (df['mean']<1), 'mean'].dropna())

In [ ]:
df.INT_TIME.value_counts()

In [ ]:
df[df.INT_TIME==14.4]['mean'].dropna().size

In [ ]:
plt.figure()
ax = sns.violinplot(x='INT_TIME', y='std', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
                   split=True, cut=0, scale='count', inner=None, bw=.2)
sns.stripplot(x='INT_TIME', y='std', hue='CHANNEL', data=df.sort_values(by='INT_TIME'),
              jitter=False, alpha=0.7, size=4)
ax.set_title("Dark STD vs INT_TIME")

In [ ]:
plt.figure()
sns.barplot(x='INT_TIME', y='mean', hue='CHANNEL', data=df.sort_values(by='INT_TIME'))

In [ ]:
plt.figure()
ax = sns.countplot(x='INT_TIME', data=df.sort_values(by='INT_TIME'), palette='Greens_d',
                  hue='CHANNEL')
ax.set_title('Counts of integrations per INT_TIME')

In [ ]:
df = df[df.INT_TIME.isin([1.4, 4.2, 14.4])]

In [ ]:
g = sns.PairGrid(df, x_vars=['INT_TIME'], hue='CHANNEL',
                y_vars=['mean','std'], size=3.5)
g.map(sns.violinplot)

In [ ]:


In [ ]:


In [ ]:

Compare with HK


In [ ]:
from iuvs import hk

In [ ]:
hkdb = pd.read_hdf('/home/klay6683/to_keep/HK_DB.h5', 'df')

In [ ]:
hkdb['2015'][['MUV_DET_TEMP_C']].resample('1min').plot()

In [ ]:
check_values('lya_centroid')

In [ ]:
df.sort_index(inplace=True)

In [ ]:
df.groupby(['CHANNEL', 'COLLECTION_ID','INT_TIME'])['median'].mean()

In [ ]:
%matplotlib nbagg

In [ ]:
df['mean'].max()

In [ ]:
df[df['mean']> 20].filter(regex='NAXIS.')

In [ ]:


In [ ]:
df[df['mean']> 20].PRODUCT_ID.values

In [ ]:
plt.figure()
df['mean'].plot(kind='hist', bins=100, logy=True, xlim=(0,3))

In [ ]:
l1a = io.L1AReader('mvn_iuv_l1a_apoapse-orbit01597-muvdark_20150725T184050_v02_r01.fits.gz',
                   env='production')

In [ ]:
l1a

In [ ]:
l1a.img.shape

In [ ]:
l1a.img.mean()

In [ ]:
l1a.p

In [ ]:
df.groupby(['activity','COLLECTION_ID'])['mean'].mean()

In [ ]:
cols_of_interest = df.columns.drop('anc_svn_revision  bin_tbl code_svn_revision obs_id'
                                   ' orbit_number process product_id'.upper().split())

In [ ]:
# df.CHANNEL = df.CHANNEL.map(lambda x: 0 if x=='MUV' else 1)

In [ ]:
df=df[cols_of_interest]
# df = df.select_dtypes(include=[float, int])
# df= df.drop('SW_VER NAXIS3'.split(), axis=1)

In [ ]:
df = df.dropna(how='any')

In [ ]:
df.to_hdf('/Users/klay6683/data/iuvs/dark_stuff/to_study.h5','df')

In [ ]:
df.head()

Odd things

Difference between MIRROR_DEG and MIR_DEG


In [ ]:
df['mirdeg_delta'] = df.MIRROR_DEG - df.MIR_DEG

In [ ]:
plt.figure()
df.mirdeg_delta.plot(style='.')

In [ ]: